import scrapy
from robot.items import Novel, Chapter


class HetuSpider(scrapy.Spider):
    name = 'hetu'

    start_urls = ['http://www.hetushu.com/book/index.php?state=2&page=209']
    #start_urls = ['http://www.hetushu.com/book/2/index.html']

    def parse(self, response):
        for href in response.css('.book_list .name a::attr(href)'):
            yield response.follow(href, self.parse_book)

        for href in response.css('.pre::attr(href)'):
            yield response.follow(href, self.parse)

    def parse_author(self, response):
        def extract_with_css(query):
            return response.css(query).extract_first().strip()

        yield {
            'name': extract_with_css('h3.author-title::text'),
            'birthdate': extract_with_css('.author-born-date::text'),
            'bio': extract_with_css('.author-description::text'),
        }

    def parse_book(self, res):
        sort_no = 0
        parent = None
        for x in res.css('#dir *'):
            sort_no = sort_no + 1
            tag_name = x.xpath('name()').extract_first()
            if tag_name == 'dt':
                parent = x.xpath('text()').extract_first()
            elif tag_name == 'dd':
                href = x.css('::attr(href)').extract_first()
                title = x.css('::text').extract_first()
                yield res.follow(href, self.parse_chapter, meta=dict(parent=parent, title=title, sort_no=sort_no))
            else:
                self.logger.info('unkown tag %s', tag_name)

        tags = res.css('.tag a::text').extract()
        yield Novel({
            'title': res.css('h2::text').extract_first().strip(),
            #'summary': res.xpath('string(//div[@class="intro"])').extract_first(),
            'summary': res.xpath('//div[@class="intro"]').extract_first(),
            'author': tags[0].strip(),
            'category': res.xpath("//div[contains(.//text(), '类型：')]").extract_first().replace('类型：', '').replace('<div>', '').replace('</div>', '').strip(),
            'tags': tags[2:],
            'source_url': res.url,
        })

    def parse_chapter(self, res):
        content = res.css('#content').extract_first()
        yield Chapter({
            'parent': res.meta.get('parent', None),
            'sort_no': res.meta.get('sort_no', 0),
            'title': res.meta.get('title', None),
            'source_url': res.url,
            'content': content,
        })
